package com.data.zfcg;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.*;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.html.HtmlInput;
import com.gargoylesoftware.htmlunit.html.HtmlBody;
import java.util.List;

import java.util.Scanner;  
import java.util.Timer;  
import java.util.TimerTask;

import java.text.SimpleDateFormat; 
import java.util.Date; 

import org.quartz.DisallowConcurrentExecution;
import org.quartz.Job;
import org.quartz.JobDetail;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

import  org.apache.commons.logging.Log;
import  org.apache.commons.logging.LogFactory;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.ThrowableInformation;
import org.apache.log4j.PropertyConfigurator;
import java.text.SimpleDateFormat; 

@SuppressWarnings("all")
public class myJob implements Job {
    private static Logger log = Logger.getLogger(Jxsggzy.class);  
    @Override 
    public void execute(JobExecutionContext arg0) throws JobExecutionException {  // throws Throwable
    	System.out.println("run"); 
		Connection conn = DataBaseHelper.getConnection();

		Statement stmt = null;
		String sql = null,proxyname=null;
		ResultSet rs = null;
		int proxyport=0;
		String verifyStr=null;
		// create a database connection
		// MySQL的JDBC URL编写方式：jdbc:mysql://主机名称：连接端口/数据库的名称?参数=值
		// 避免中文乱码要指定useUnicode和characterEncoding
		// 执行数据库操作之前要在数据库管理系统上创建一个数据库，名字自己定，
		// 下面语句之前就要先创建Jxsggzy数据库
		if (conn != null) {

		try{									
			stmt = conn.createStatement(); //通过连接创建(createStatement)一个语句对象,不是new 
			rs = stmt.executeQuery("select * from ipliststemp order by verifynum desc,speed asc;");   //执行查询,返回一个结果集rs相当于一个游标 
			while (rs.next()) { //循环取得结果集(遍历) 			
		    	proxyname=rs.getString("ipname");
				proxyport = rs.getInt("port");
				String testurl = rs.getString("reserve");
		    	System.out.println("testurl:"+testurl+" ipname:"+proxyname+" portname:"+proxyport);
		    	try{
		    	String code=getConnStatus(testurl,proxyname,proxyport);		    	
		      if(code.startsWith("2")){
		            	System.out.println(" ok!");
				    	verifyStr=webclientproxytest(testurl,proxyname,proxyport);
				    	if(verifyStr.length()>0&&verifyStr.indexOf("not")<1){
			            	System.out.println("while verifynum ok:"+verifyStr.length());
		            	break;}
		        }
  			} catch(Exception e){
  		        System.out.println("ipliststemp while :"+proxyname+":" + proxyport + " is not aviable");//异常IP
  				}
		      
				}

			rs = stmt.executeQuery("select * from jxsggzy"); //执行查询,返回一个结果集rs相当于一个游标
			while (rs.next()) { //循环取得结果集(遍历) 			
	    	String typename=rs.getString("typename");
			String tablename = rs.getString("tablename");
			String areaname=rs.getString("areaname");
			String cityname=rs.getString("cityname");
	        String baseurl = rs.getString("baseurl");
	        String url = rs.getString("url");
	        //try{
	    	//String code=getConnStatus(url,proxyname,proxyport);
	       //if(code.startsWith("2")){
	            	System.out.println("tablename:"+tablename+" areaname:"+areaname+" citynname:"+cityname+" typenname:"+typename+" ok!");
	                gettitle(tablename,areaname,cityname,typename,baseurl,url,proxyname,proxyport);
	       // } 
  			//} catch(Exception e){
  		      //  System.out.println("jxsggzy while :"+proxyname+":" + proxyport + " is not aviable");//异常IP
  				//}	       	       
			}
			
			
			System.out.println("全部采集完成！");
			
			
			/*
    	for(int bidclass=0;bidclass<= 10; bidclass++){ //int bidclass=4;bidclass<= 10;
	    String bidclassname = String.format("%02d", bidclass);   
    	String table_class = new IntToStrong().getname(Integer.parseInt(bidclassname));
    	for(int table_number=0;table_number<= 4; table_number++)
    	{   		
    	String typename=table_class+table_number;
		String table_name = table_class+table_number;
		String areaname="jx";
		if (table_name=="zfcg1" || table_name=="zfcg2" || table_name=="zfcg4"){table_name="zfcg0";}
            String baseurl = "http://ggzy.jiangxi.gov.cn";
            String url = baseurl+"/jxzbw/jyxx/0020"+bidclassname+"/0020"+bidclassname+"00"+ (table_number + 1) + "/MoreInfo.aspx?CategoryNum=0020"+bidclassname+"00" + (table_number + 1);
    		String code=getConnStatus(url);
            if(code.startsWith("2")){
            	System.out.println(typename+" ok!");
                gettitle(table_name,areaname,typename,baseurl,url);
            }    
    	}
    	}
    	*/
		rs.close();
		stmt.close();
		conn.close();
	  } catch(Exception e){
		// TODO Auto-generated catch block
		e.printStackTrace();
        StringWriter sw = new StringWriter();   
        e.printStackTrace(new PrintWriter(sw, true));   
        String strs = sw.toString(); 
        log.error("start error:"+ strs);
        }
		finally{
			try{
			    if(rs != null){
			        rs.close();
			        rs = null;
			    }  
			}catch(SQLException e){
			    e.printStackTrace();
			}finally{
			    try{
			        if(stmt != null){
			        	stmt.close();
			        	stmt = null;
			   }
			    }catch(SQLException e){
			        e.printStackTrace();
			    }finally{
			        try{
			     if(conn != null){
			   conn.close();
			   conn = null;
			     }
			}catch(SQLException e){
			     e.printStackTrace();
			}finally{
				System.out.println("数据库连接已关闭！");
			}
			    }
			}
		}
	}
		
	}
    
    public static void gettitle(String tablename,String areaname,String cityname,String typename,String baseurl,String url,String proxyname,int proxyport) 
    {
        String[] Pagecountsp = null; 
        boolean tmptable=true;
    	try{
    		
        //WebClient webClient = new WebClient(BrowserVersion.CHROME,"45.63.121.157",80);
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
       // final DefaultCredentialsProvider credentialsProvider = (DefaultCredentialsProvider) webClient.getCredentialsProvider();
      //  credentialsProvider.addCredentials("username","password");
        
        ProxyConfig proxyConfig = new ProxyConfig(proxyname,proxyport);  
        //proxyConfig.setProxyHost("192.168.XX.XX");
		//proxyConfig.setProxyPort(8080);
        webClient.getOptions().setProxyConfig(proxyConfig);  
        
        final IncorrectnessListener il = new MyIncorrectnessListener();
        webClient.setIncorrectnessListener(il);        
        webClient.getOptions().setTimeout(200000);
    	webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        webClient.getOptions().setThrowExceptionOnScriptError(false);  
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false); 
        webClient.setJavaScriptTimeout(200000);
        //webClient.waitForBackgroundJavaScript(30000);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
    	DataBaseHelper dbh = new DataBaseHelper(tablename);
    	dbh.onCreate(tablename);
    	String tmp_tablename=dbh.onCreatetmp(tablename);
    	//System.out.println("url:"+url);
    	HtmlPage page=null;
    	try{
        page = (HtmlPage)webClient.getPage(url);
			} catch(Exception e){
  		        System.out.println("(HtmlPage)webClient "+proxyname+":" + proxyport + " is not aviable");//异常IP
  				}        
    	if(page!=null){
        Document doc = Jsoup.parse(page.asXml());
        Elements ele = doc.select("div#MoreInfoList1_Pager").select("td");
        String Pagecount=ele.text(); 
        Pagecountsp = Pagecount.split(" ");  
        if (Pagecountsp.length>3){
        int PageCou = Integer.parseInt(Pagecountsp[3]); //页数
        int RecordCou = Integer.parseInt(Pagecountsp[1]); //记录总数
        //log.info(Pagecountsp[0]+"TT"+Pagecountsp[1]+"TT"+Pagecountsp[2]+"TT"+Pagecountsp[3]+"TT"+Pagecountsp[4]);
    	int DBcout=dbh.DBcount(tablename,typename);
    	System.out.println(" PageCou:"+PageCou+" RecordCou:"+RecordCou+" DBcount:"+DBcout);
    	if(RecordCou>0 && DBcout==0){    	     //新表,没采集过
    		System.out.println("新表,没采集过");
    	getListData(dbh,baseurl,doc,tmp_tablename,areaname,cityname,typename, PageCou,RecordCou);  
    		int PageCount=PageCou;
    		PageCou--;
    	for (int i = 2; i <= PageCount; i++,PageCou--) 
    		{     		
    		if (RecordCou>60 ){RecordCou=RecordCou-60;}
    		System.out.println("总页数："+PageCount+" 现在页数："+PageCou+" 跳转页："+i);
            page=ListData(webClient,page,i,tmp_tablename,url);
            doc = Jsoup.parse(page.asXml());            
        	getListData(dbh,baseurl,doc,tmp_tablename,areaname,cityname,typename, PageCou,RecordCou);
    		}
    	}
    	else if(RecordCou>=(DBcout+60)){  //记录数大于数据库条目，未在当页即60内，完全新增，建立tmp增量库
    		System.out.println("记录数大于数据库条目，未在当页即60内，新增");
        	getListData(dbh,baseurl,doc,tmp_tablename,areaname,cityname,typename,PageCou,RecordCou);  
    		int PageCount=PageCou;
    		PageCou--;
    	for (int i = 2; i <= PageCount; i++,PageCou--) 
    		{     		
    		if (RecordCou>60 ){RecordCou=RecordCou-60;}
    		if(RecordCou>=(DBcout+60)){  //记录数大于数据库条目，未在当页即60内，完全新增
    			System.out.println("总页数："+PageCount+" 现在页数："+PageCou+" 跳转页："+i);
            page=ListData(webClient,page,i,tmp_tablename,url);
            doc = Jsoup.parse(page.asXml());            
        	getListData(dbh,baseurl,doc,tmp_tablename,areaname, cityname,typename, PageCou,RecordCou);
    		}
    		else	if(RecordCou>DBcout && RecordCou<=(DBcout+60)){	//记录数大于数据库条目，在当页即60内，比较新增
    			System.out.println("记录数大于数据库条目，在当页即60内，比较新增");
    			getRemainData(dbh,baseurl,doc,tmp_tablename,areaname,cityname,typename, PageCou,RecordCou);
    			i=PageCount; //比较完成后提前结束
    		}
        	else if(DBcout == RecordCou){  //刚好完成，完全相同
        		
        	}
    		}
    	}
    	else if(RecordCou>DBcout && RecordCou<=(DBcout+60)){	
    		System.out.println("记录数大于数据库条目，在当页即60内，比较新增");//记录数大于数据库条目，在当页即60内，比较新增,建立tmp增量库
    		getRemainData(dbh,baseurl,doc,tmp_tablename, areaname,cityname,typename,PageCou,RecordCou);
    	}	
    	else if(DBcout == RecordCou){  //已经采集，完全相同
    		tmptable=false;
    		System.out.println("已经采集，完全相同！");    		
    	}
    }
        //dbh.Deduplication(tmp_tablename,areaname,typename,cityname);
        if(tmptable){
        dbh.Dedup_Add_table(tablename,tmp_tablename,areaname,typename,cityname,tmptable);
        }
    	}
        
        
    	webClient.close();
    	
    	
    	
  	  } catch(Exception e){
  		// TODO Auto-generated catch block
  		e.printStackTrace();
  		  
         StringWriter sw = new StringWriter();   
         e.printStackTrace(new PrintWriter(sw, true));   
         String strs = sw.toString(); 
         log.error("url:"+url+"gettitle error:"+ strs);
          }
    }


	static HtmlPage ListData(WebClient webClient,HtmlPage page,int pageNo,String table_name,String url) {
    	String doPostBackPage="javascript:__doPostBack('MoreInfoList1$Pager','"+pageNo+"')";   
    	try{
    	HtmlAnchor loginLink = page.getAnchorByHref(doPostBackPage);
    	HtmlPage resultPage  = loginLink.click();
    	webClient.waitForBackgroundJavaScript(1000*3L);
    	//webClient.waitForBackgroundJavaScriptStartingBefore(1000L);
        page=resultPage;
        //waitForLoad(resultPage);              
        //webClient.closeWindows();
       return page;       
	  } catch(Exception e){
		// TODO Auto-generated catch block
		e.printStackTrace();
		  
        StringWriter sw = new StringWriter();   
        e.printStackTrace(new PrintWriter(sw, true));   
        String strs = sw.toString(); 
        log.error("table_name:"+table_name+"url:"+url+"ListData error:"+ strs);
		return page;
        }
    }
    
    public static void getListData(DataBaseHelper dbh,String baseurl, Document doc,String tablename,String areaname,String cityname,String typename,int Pageno, int recordCou) throws Exception  
    {
    	float budget=0;
    	List<Map<String, Object>> newslist = new ArrayList<Map<String, Object>>();
    	Map<String, Object> map = new HashMap<String, Object>();
        int newsclass=recordCou;
    	Element ele = doc.select("td[height=500]").first();
    	try{
    	if (ele != null){
    	Elements eles = ele.select("a");
        for (Element ele2 : eles) {
            String newstitle = ele2.text().trim();
            String newsurl = baseurl+ele2.attr("href").trim();
            String newstime = ele2.parent().parent().select("td").last().text().trim();
            if (ele2.text().replace("更多信息", "").length() >= 2) {
                String newsdetailstext=getNews(newsurl).trim();
                String newsdetailshtml=null; //String newsdetailshtml=getNewshtml(newsurl).trim();
                budget=getbudget(newsurl,newsdetailstext);  //需要对文本分析，是否需要*万
            	map.put("pageno", Pageno);
                map.put("news_class",newsclass);
                map.put("areaname",areaname);
                map.put("typename",typename);
                map.put("news_title",newstitle);
                map.put("news_time", newstime);
                map.put("isexpire", "Null");
             	map.put("news_url", newsurl);
             	map.put("budget", budget);
             	map.put("newsdetailstext", newsdetailstext); 
             	map.put("newsdetailshtml", newsdetailshtml); 
            	log.info("newsurl:"+newsurl);
             	newslist.add(map);

             	dbh.addnewslist(tablename,Pageno,newsclass,areaname,cityname,typename,newstitle,newstime,"NULL",newsurl,budget,newsdetailstext,newsdetailshtml);
              if(newsclass>=0){newsclass--;}
            }
        }
        //dbh.addmaplist(table_name,newslist);
    	}
  	  } catch(Exception e){
  		// TODO Auto-generated catch block
  		e.printStackTrace();
          StringWriter sw = new StringWriter();   
          e.printStackTrace(new PrintWriter(sw, true));   
          String strs = sw.toString(); 
          log.error("start error:"+ strs);
          }
    } 
    
    public static void getRemainData(DataBaseHelper dbh,String baseurl, Document doc,String tablename, String areaname,String cityname,String typename,int Pageno, int recordCou)  //60页内的比较增加，相同title不写入，直到RECORD与DBCOUT相同  
    {
    	List<Map<String, Object>> newslist = new ArrayList<Map<String, Object>>();
        Map<String, Object> map = new HashMap<String, Object>();
        int newsclass=recordCou;
        float budget=0;
    	Element ele = doc.select("td[height=500]").first();
    	try{
    	if (ele != null){
    	Elements eles = ele.select("a");
        for (Element ele2 : eles) {
            String newstitle = ele2.text();
            String newsurl = baseurl+ele2.attr("href");
            String newstime = ele2.parent().parent().select("td").last().text();
            if (ele2.text().replace("更多信息", "").length() >= 2) { //存在重复不进行插入
            	if (dbh.Comparison(tablename,typename,newstitle,newstime,newsurl)) {
            	log.info("60条内，新增开始。。。");
                String newsdetailstext=getNews(newsurl).replace("Null", "");
                String newsdetailshtml=getNewshtml(newsurl).trim();
                //log.info("getbudget:start");
                budget=getbudget(newsurl,newsdetailstext);

            	map.put("pageno", Pageno);
            	map.put("news_class",newsclass);
             	map.put("news_title",newstitle);
             	map.put("news_time", newstime);
             	map.put("isexpire", "Null");
             	map.put("news_url", newsurl);
             	map.put("budget", budget);
             	map.put("newsdetailstext", newsdetailstext);
             	map.put("newsdetailshtml", newsdetailshtml); 
             	dbh.addnewslist(tablename,Pageno,newsclass,areaname,typename,cityname,newstitle,newstime,"Null",newsurl,budget,newsdetailstext,newsdetailshtml);
              newslist.add(map);
              if(newsclass>=0){newsclass--;}
            	}
            }
        } 
    	}
    	  } catch(Exception e){
    	  		// TODO Auto-generated catch block
    	  		e.printStackTrace();
    	          StringWriter sw = new StringWriter();   
    	          e.printStackTrace(new PrintWriter(sw, true));   
    	          String strs = sw.toString(); 
    	          log.error("start error:"+ strs);
    	          }
    }
    
    static String getNewshtml(String newsurl) {
    	String newsdetailshtml="Null"; 
    	Document doc = null;
    	Response resp=null;
    	try{
    	org.jsoup.Connection con = Jsoup.connect(newsurl).userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21");
    	con.timeout(20000).ignoreHttpErrors(true).followRedirects(true);
    	try{
    	resp = con.execute();
        doc = con.get();
		} catch(Exception e){
	        System.out.println("getbudget resp "+newsurl+" is not aviable");//异常IP
			}    
    	if (resp!=null&&resp.statusCode() == 200) {
        Element ele = doc.getElementById("TDContent");
        int j = 1;
        if(ele !=null){
        	newsdetailshtml=ele.toString();
        	//newsdetailshtml =URLEncoder.encode(ele.toString(), "UTF-8");       	
        }
    	}
        return newsdetailshtml;
  } catch(Exception e){
	// TODO Auto-generated catch block
	e.printStackTrace();
    StringWriter sw = new StringWriter();   
    e.printStackTrace(new PrintWriter(sw, true));   
    String strs = sw.toString(); 
    log.error("start error:"+ strs);
    return newsdetailshtml;
    }
}

    static String getNews(String newsurl) {
        	String newsdetailstext="Null"; 
        	Document doc = null;
        	Response resp=null;
        	try{
        	org.jsoup.Connection con = Jsoup.connect(newsurl).userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21");
        	con.timeout(18000).ignoreHttpErrors(true).followRedirects(true);
        	try{
        	resp = con.execute();
            doc = con.get();
    		} catch(Exception e){
		        System.out.println("getNews resp "+newsurl+" is not aviable");//异常IP
				}     
        	
        	if (resp!=null&&resp.statusCode() == 200) {
            Element ele = doc.getElementById("TDContent");
            int j = 1;
            if(ele !=null){
                if (ele.children().text().length() < 2) {
              	  if(ele.text().trim().length()>0){
                     newsdetailstext = newsdetailstext + ele.text().trim();}
                 }
                 for (Element ele2 : ele.children()) {
                     j++;
                     newsdetailstext = newsdetailstext + ele2.text().trim();
                     if(ele2.text().trim().length()>0){
                     newsdetailstext = newsdetailstext + "\n";
                     newsdetailstext = newsdetailstext + "\n";}
                 }
                 if (j == 2) {
                     newsdetailstext = "";
                     for (Element ele2 : ele.children()) {
                         for (Element ele3 : ele2.children()) {
                             newsdetailstext = newsdetailstext + ele3.text().trim();
                             if(ele3.text().trim().length()>0){
                             newsdetailstext = newsdetailstext + "\n";
                             newsdetailstext = newsdetailstext + "\n";}
                         }
                     }
                 }  
                 }
        	}
            return newsdetailstext;
	  } catch(Exception e){
		// TODO Auto-generated catch block
		e.printStackTrace();
        StringWriter sw = new StringWriter();   
        e.printStackTrace(new PrintWriter(sw, true));   
        String strs = sw.toString(); 
        log.error("start error:"+ strs);
        return newsdetailstext;
        }
    }
    
    
    static float getbudget(String newsurl,String newsdetailstext) {     	
        float budget=0;
        boolean wan = false,last=false;
        Response resp=null;
    	Document doc = null;
        try{
    	org.jsoup.Connection con = Jsoup.connect(newsurl).userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21");
    	con.timeout(18000).ignoreHttpErrors(true).followRedirects(true);
    	try{
    	resp = con.execute();
        doc = con.get();
		} catch(Exception e){
	        System.out.println("getbudget resp "+newsurl+" is not aviable");//异常IP
			}     
    	if (resp!=null&&resp.statusCode() == 200) {
        //Element ele = doc.getElementById("TDContent");
        Element infoTable = doc.getElementsByAttributeValue("class", "MsoNormalTable").first();//获得信息表数据
        if(infoTable ==null){
        	infoTable = doc.getElementsByAttributeValue("class", "MsoTableGrid").first();
        	}
        /*
         * Elements tableLineInfos = infoTable.select("tr"); //对该信息表继续进行筛选，获得一个tr 集合 数据        
        for (Element lineInfo : tableLineInfos) {
            String lineInfoContent = lineInfo.select("td").last().text().trim(); //获得tr集合中的一个td元素
            int result= lineInfoContent.indexOf("预算");
            if(result>0){
            log.info("jsoup is :" + lineInfoContent);
            }
            }
            */        
        if(infoTable !=null){
        	log.info("infoTable抽取开始");
        Elements trs = infoTable.select("tr");
        boolean isfirst=true;
        outer:
        for(int i = 0;i<trs.size();i++){
        	log.info("trs.size()"+trs.size());
            Elements tds = trs.get(i).select("td");
            inner:
            for(int j = 0;j<tds.size();j++){
            	
            	String text1="";
            	String lasttext1="";
                String text = tds.get(j).text().trim();    
                int jj=0;
               // log.info("i:"+i+"j:"+j+":"+text);
                
                if(text.indexOf("预算")>=0||text.indexOf("合计")>=0||(iswan(text))||text.indexOf("元")>=0||text.indexOf("金额")>=0){
                	lasttext1=tds.get(tds.size()-1).text().trim();
                	//log.info("lasttext1:"+lasttext1);
                	if(isfirst&&(lasttext1.indexOf("预算")>=0||lasttext1.indexOf("合计")>=0||iswan(lasttext1)==true||lasttext1.indexOf("元")>=0||lasttext1.indexOf("金额")>=0)){
                		j=tds.size()-1;
                		text=lasttext1;                		
                		isfirst=false;
                		   if((Strtofloat(text)>1.0)&&((text.indexOf(".0")>0)||(text.indexOf("万元")>0)))
                           {
                			   boolean wan0=iswan(text);
                			   boolean wan1=iswan(text1);
                           if((text.indexOf(".00元")==-1)&&(wan0==true||wan1==true)){  
                        	   
                               budget=Strtofloat(text)*10000;
                               if((newsdetailstext.indexOf("县")>0||newsdetailstext.indexOf("市")>0)&&(newsdetailstext.indexOf("电脑")>0||
                        			   newsdetailstext.indexOf("摄像机")>0)&&budget>100000000){  //县城电脑等超过1亿
                            	budget=budget/10000;
                               }
                               if(budget>999999900){budget=budget/10000;}  //大于9.9个亿，网站错误金额
                               }
                               else{
                           	budget=Strtofloat(text);}
                          // log.info("ok!"+"text1："+text1+"i"+i+"j:"+j+"budget"+budget);
  							if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿，2万-10亿
  								log.info("数据正常" + budget);
	                           	break outer;
   							} else {
   								log.info("数据异常," + budget + "，结果清0");
   								budget = 0;
   							}	
                               }
                	}                	
                	int jjj=(trs.size()-1)<1?(trs.size()-1):1;
                	//log.info("newsurl:"+newsurl+"i:"+i+"j:"+j+"jjj"+jjj+"trs.size()-1):"+(trs.size()-1)+"text :" + text);
                	//log.info("trs.get(jjj).select(td).size():"+trs.get(jjj).select("td").size()+"trs.get(0).select(td).size():"+trs.get(0).select("td").size()+":trs.get(trs.size()-1).select(td).size()"+trs.get(trs.size()-1).select("td").size());
                if((trs.get(0).select("td").size()==trs.get(trs.size()-1).select("td").size()))//&&(trs.get(1).select("td").size()>j)
                {
            		//log.info("trs.get(trs.size()-1).select(td).size() :" + trs.get(trs.size()-1).select("td").size()+"j:"+j);
            	if(trs.get(trs.size()-1).select("td").size()<=j)
            	{jj=trs.get(trs.size()-1).select("td").size()-1;}
            	else{jj=j;}
            	if((i+1)<trs.size()&&jj>=0&&jj<trs.get(i+1).select("td").size()){text1=trs.get(i+1).select("td").get(jj).text().trim(); }  
            	else if((i+1)<trs.size()&&jj>=0&&(trs.get(i+1).select("td").size()-1)>0){text1=trs.get(i+1).select("td").get(trs.get(i+1).select("td").size()-1).text().trim(); } 
                	//log.info("budget :" + text1+"i:"+1+"jj:"+jj);
                }
                else //
                	if(i+1<trs.size()){
                		//log.info("trs.get(i+1).select(td).size() :" + trs.get(i+1).select("td").size()+"j:"+j);
                	if(trs.get(i+1).select("td").size()<=j)
                	{jj=trs.get(i+1).select("td").size()-1;}
                	else{jj=j;}
               if(jj>=0) {text1=trs.get(i+1).select("td").get(jj).text().trim(); }
                
               // log.info("budget :" + text1+"i:"+1+"jj:"+jj);
                }                
                
                if((Strtofloat(text1)>1.0)&&((text1.indexOf(".0")>0)||iswan(text1)==true||(text1.indexOf("元")>0)||iswan(text)==true||(text.indexOf("元")>0)))
                {
      			   boolean wan0=iswan(text);
      			   boolean wan1=iswan(text1);
                 if((text.indexOf(".00元")==-1&&text1.indexOf(".00元")==-1)&&(wan0==true||wan1==true)){                                    
                    budget=Strtofloat(text1)*10000;
                    if((newsdetailstext.indexOf("县")>0||newsdetailstext.indexOf("市")>0)&&(newsdetailstext.indexOf("电脑")>0||
             			   newsdetailstext.indexOf("摄像机")>0)&&budget>100000000){  //县城电脑等超过1亿
                 	budget=budget/10000;
                    }
                    if(budget>999999900){budget=budget/10000;}  //网站错误金额
                    }
                    else{
                	budget=Strtofloat(text1);}
                //log.info("ok!"+"text1："+text1+"i"+i+"j:"+j+"budget"+budget);
                
					if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿，2万-10亿
						log.info("数据正常" + budget);
                       	break outer;
						} else {
							log.info("数据异常," + budget + "，结果清0");
							budget = 0;
						}	
					
                    }
                
                //扫描最后一行第一个TD是否有合计
                if(trs.get(trs.size()-1).select("td").size()>0){
                String total = trs.get(trs.size()-1).select("td").get(0).text().trim();  
                log.info("total:" +total);              
                if(total.indexOf("合计")>=0||total.indexOf("元")>=0||total.indexOf("金额")>=0||(iswan(text))||(text.indexOf("元")>0))
                	{
                	if((total.indexOf("元")>0||total.indexOf(".0")>0)&&Strtofloat(total)>0){
                	text1 =total; 
                	//log.info("找到合计:" +text1);
                	}
                	else{
                    //扫描最后一行最后一个TD是否有合计
                    String total1 = trs.get(trs.size()-1).select("td").get(trs.get(trs.size()-1).select("td").size()-1).text().trim();  
                   // log.info("total:" +total1);              
                    if((total1.indexOf("元")>0||total1.indexOf(".0")>0)&&Strtofloat(total1)>0){
                    	text1 =total1; 
                    	log.info("找到最后一个合计:" +text1);
                    }
                	}
                }                
                }
                
                if(Strtofloat(text1)<10000&&last==false&&trs.size()>3){
	                if(trs.get(trs.size()-2).select("td").size()>0){
		                String total = trs.get(trs.size()-2).select("td").get(0).text().trim();  
		                //log.info("total:" +total);              
		                if(total.indexOf("合计")>=0||total.indexOf("元")>=0||total.indexOf("金额")>=0||(iswan(text))||(text.indexOf("元")>0))
		                	{
		                	if((total.indexOf("元")>0||total.indexOf(".0")>0)&&Strtofloat(total)>0){
		                	text1 =total; 
		                	log.info("找到倒数第二行合计:" +text1);
		                	last=true;
		                	}		                	
		                	else{
		                    //扫描最后一行最后一个TD是否有合计
		                    String total1 = trs.get(trs.size()-2).select("td").get(trs.get(trs.size()-2).select("td").size()-1).text().trim();  
		                    log.info("total:" +total1);              
		                    if((total1.indexOf("元")>0||total1.indexOf(".0")>0)&&Strtofloat(total1)>0){
		                    	text1 =total1; 
		                    	log.info("找到倒数第二行一个合计:" +text1);
		                    	last=true;
		                    }
		                	}
		                }
		                }
                	
                }
            
                if(text1.indexOf(".00元")==-1&&iswan(text1)==true){
                	wan=true;
                }
               // log.info("万:text1:" +text1+"text:"+text+"text1.length()"+text1.length());
                if(text1.length()>0&&wan&&22>text1.length()){
         		   budget =Strtofloat(text1)*10000;   
                   if((newsdetailstext.indexOf("县")>0||newsdetailstext.indexOf("市")>0)&&(newsdetailstext.indexOf("电脑")>0||
            			   newsdetailstext.indexOf("摄像机")>0)&&budget>100000000){  //县城电脑等超过1亿
                	budget=budget/10000;
                   }
         		  if(budget>999999900){budget=budget/10000;}  //大于9.9个亿，网站错误金额
         	   }
         	   else{
         		   budget =Strtofloat(text1);  
         	   }
                log.info("budget is:" +budget);
					if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿，2万-10亿
						log.info("数据正常" + budget);
                       	break outer;
						} else {
							log.info("数据异常," + budget + "，结果清0");
							budget = 0;
						}	
                }
            }

        }
        
    	}
        if(budget==0.0){
        	boolean wan4=false;
        	Element ele = doc.getElementById("TDContent");
            if(ele !=null){
            	String detailtext=ele.text();
                if(detailtext.indexOf("本项目投资")>0){ 
                	//log.info("本项目投资"+detailtext.indexOf("本项目投资"));
                	String detailtext1=detailtext.substring(detailtext.indexOf("本项目投资"));
                	//log.info("万元"+detailtext1.indexOf("万元"));
                	if(detailtext1.indexOf("万元")>0&&detailtext1.indexOf("万元")<25){
                    	String textok = detailtext1.substring(0, detailtext1.indexOf("万元")).trim();
	                	//log.info("textok"+textok);
                    	budget=Strtofloat(textok)*10000;
                        if((newsdetailstext.indexOf("县")>0||newsdetailstext.indexOf("市")>0)&&(newsdetailstext.indexOf("电脑")>0||
                 			   newsdetailstext.indexOf("摄像机")>0)&&budget>100000000){  //县城电脑等超过1亿
                     	budget=budget/10000;
                        }
                    	if(budget>999999900){budget=budget/10000;}  //大于9.9个亿，网站错误金额
							if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿，2万-10亿
								log.info("数据正常" + budget);
   							} else {
   								log.info("数据异常," + budget + "，结果清0");
   								budget = 0;
   							}	
                	}
                    }
                if(budget==0.0&&detailtext.indexOf("合计")>0){ 
                	//log.info("合计"+detailtext.indexOf("合计"));
                	String detailtext1=detailtext.substring(detailtext.indexOf("合计"));
                	//log.info(".00元"+detailtext1.indexOf(".00元"));
                	if(detailtext1.indexOf(".00元")>0&&detailtext1.indexOf(".00元")<25){
                    	String textok = detailtext1.substring(0, detailtext1.indexOf(".00元")).trim();
	                	log.info("textok"+textok);
                    	budget=Strtofloat(textok);
                    	if(budget>999999900){budget=budget/10000;}  //大于9个亿，网站错误金额
							if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿
								log.info("数据正常" + budget);
							} else {
								log.info("数据异常," + budget + "，结果清0");
								budget = 0;
							}	
                	}
                    }
                if(budget==0.0&&detailtext.indexOf("成交总金额")>0){ 
                	log.info("成交总金额"+detailtext.indexOf("成交总金额"));
                	String detailtext1=detailtext.substring(detailtext.indexOf("成交总金额"));
                	log.info(".00"+detailtext1.indexOf(".00"));
                	if(detailtext1.indexOf(".00")>0&&detailtext1.indexOf(".00")<33){
                    	String textok = detailtext1.substring(0, detailtext1.indexOf(".00")).trim();
	                	log.info("textok"+textok);
                    	budget=Strtofloat(textok);
                    	if(budget>999999900){budget=budget/10000;}  //大于9个亿，网站错误金额
							if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿
								log.info("数据正常" + budget);
							} else {
								log.info("数据异常," + budget + "，结果清0");
								budget = 0;
							}	
                	}
                    }
                
                if(budget==0.0){
        	log.info("TDContent抽取开始");
                Elements trs = ele.select("tr");
                outer:
                for(int i = 0;i<trs.size();i++){
                	//log.info("trs.size()"+trs.size());
                    Elements tds = trs.get(i).select("td");
                    inner:
                    for(int j = 0;j<tds.size();j++){
                    	//log.info("tds.size()"+tds.size());
                    	String text1="";
                        String text = tds.get(j).text().trim();         
                        //log.info("text："+text+"i"+i+"j:"+j);

                		if(text.indexOf("万元")>-1){wan4=iswan(text);}
                        if(text.indexOf(".00")>=0||text.indexOf("预算")>=0||text.indexOf("合计")>=0||iswan(text)==true||text.indexOf("元")>=0||text.indexOf("金额")>=0){
                        	if(Strtofloat(text)>0)
                        	{
                        		boolean wan3=iswan(text);
                                if((wan3==true||wan4==true)){                   //text.indexOf(".00")==-1&&               
                                	budget=Strtofloat(text)*10000;
                                    if((newsdetailstext.indexOf("县")>0||newsdetailstext.indexOf("市")>0)&&(newsdetailstext.indexOf("电脑")>0||
                             			   newsdetailstext.indexOf("摄像机")>0)&&budget>100000000){  //县城电脑等超过1亿
                                 	budget=budget/10000;
                                    }
                                	if(budget>999999900){budget=budget/10000;}  //大于9个亿，网站错误金额
                                }
                                else{
                        		budget=Strtofloat(text);}
                               // log.info("555ok!"+"text："+text+"i"+i+"j:"+j+"budget"+budget);
      							if ((budget != 1.0 || budget != 10000.0) && budget >= 20000.0 && (budget < 2147483647)) { // 整型最大值：2147483647,214亿，2万-10亿
      								log.info("数据正常" + budget);
    	                           	break outer;
       							} else {
       								log.info("数据异常," + budget + "，结果清0");
       								budget = 0;
       							}	
                        	}                        	
                        }
                    }
                }
            }
             }              
        }

    	}  
        return budget;
  	  } catch(Exception e){
  		// TODO Auto-generated catch block
  		e.printStackTrace();
          StringWriter sw = new StringWriter();   
          e.printStackTrace(new PrintWriter(sw, true));   
          String strs = sw.toString(); 
          log.error("newsurl:"+newsurl+"getbudget error:"+ strs);
          return budget;
          }
}
    
    static String Strtonum(String text)
    {
    char[] b = text.toCharArray();
    String result ="";
    String result1="";
    try{
    for (int i = 0; i < b.length; i++)
    {
    if (("0123456789.").indexOf(b[i] + "") != -1)
    {
    result += b[i];
    }
    }
    return result1;
	  } catch(Exception e){
	  		// TODO Auto-generated catch block
	  		e.printStackTrace();
	          StringWriter sw = new StringWriter();   
	          e.printStackTrace(new PrintWriter(sw, true));   
	          String strs = sw.toString(); 
	          log.error("Strtonum error:"+ strs);
	          return result1;
	          }
    }
    
    static boolean iswan(String text) {
		char[] b = text.toCharArray();
		boolean result1 = false;
		int start=0,end=0;
		try {
			if(text.indexOf("万")>0&&text.indexOf("整（")==-1&&text.indexOf("万元整 ￥")==-1){ //&&text.indexOf("万元整（")==-1
				start=text.indexOf("万")+1;
				end=text.length();
				if(start<end&&(b[start]==32||b[start]==160||b[start]=='\n'||b[start]=='元'||b[start]==12288)){
					result1=true;
				}
				if(start==end){result1=true;}
   				//System.out.println("wan:"+text.indexOf("万")+" "+text.length());
   				if(text.indexOf("万")<text.length()&&text.indexOf("万")!=0){  //判断万是否为单元，还是大写，避免提取大写中的单元
   					//System.out.println("b:"+b[text.indexOf("万")-1]+"is:"+("零壹贰叁肆伍陆柒捌玖拾佰仟万").indexOf(b[text.indexOf("万")-1]));
   					if(("零壹贰叁肆伍陆柒捌玖拾佰仟万").indexOf(b[text.indexOf("万")-1]) != -1){result1=false;}
   					
   				}
				
			}
			return result1;
		} catch (Exception e) {
			e.printStackTrace();
			StringWriter sw = new StringWriter();
			e.printStackTrace(new PrintWriter(sw, true));
			String strs = sw.toString();
			log.error("iswan error:" + strs);
			return result1;
		}
	}
    
    static float Strtofloat(String text) throws Exception
    {
     	 String result ="";//,tmp="";
       float result1=0;
       int isInteger=0,num=0;
       boolean preint=false,isfloat=false;
       try {
  			text=text.replace("．", ".");
   			text=text.replace(",", "");
   			text=text.replace("，", "");
   			if(text.indexOf("/")>0){text=text.substring(0, text.indexOf("/"));} //672/3年
  	        char[] b = text.toCharArray();
  			String[] nums = text.split("\\D+");
  			//log.info("nums.length:"+nums.length);
  			if(text.indexOf('.')>-1&&nums.length<4){isfloat=true;}  //含小数点，有3段，如222.22
  			if(text.indexOf('.')==-1&&nums.length<3){isfloat=true;}  //含小数点，有3段，如222.22
  			for (int i = 0; i < b.length&&text.length()<40&&isfloat; i++) //壹亿壹仟壹佰壹拾玖万壹仟陆佰肆拾元整（￥：191640.00元）
  			{
  				if(preint==false&&(b[i]=='0'||b[i]=='.')){isfloat=false;log.info("0和.在数字前面"+i+"b[i]"+b[i]);}
  			if (("0123456789.").indexOf(b[i] + "") != -1)
  			{
  			result += b[i];
  			preint=true;
  			if(b[i] == '.')
  			{
  			   num++;
  			}
  			
  			}
  			else if (preint==true){  				
  				isInteger++;
  				preint=false;	
  				if(b[i]=='B'){result="";break;}
  			}						
  			}
  			//log.info("result"+result);
  			//result = result.substring(result.lastIndexOf('.', result.lastIndexOf('.') - 1) + 1);
  			//log.info("result"+result);
  			//log.info("result"+result+"result.indexOf(.)="+result.indexOf(".")+"num:"+num+"isInteger"+isInteger);
  			if(result.length()>0&&20>result.length()&&num<2&&isInteger<3&&result.indexOf(".")!=0){ //,检测是否中间为正常浮点数如：xxx123.12xxx
  			result1 =Float.parseFloat(result); 
  			}
  			if(text.indexOf("万m3")>0)
  			{result1=0;}
  			//log.info("result1:"+result1);
  	    return result1;
  	    
  	  } catch(Exception e){
  	  		// TODO Auto-generated catch block
  	  		e.printStackTrace();
  	          StringWriter sw = new StringWriter();   
  	          e.printStackTrace(new PrintWriter(sw, true));   
  	          String strs = sw.toString(); 
  	         log.error("Strtofloat error:"+ strs);
  	          return result1;
  	          }
}
    
	public static String getConnStatus(String url,String proxyname,int proxyport) throws IOException{

		 String code="400";
         URL hurl = null;
		try {
			hurl = new URL(url);
		} catch (MalformedURLException e2) {
			// TODO Auto-generated catch block
			e2.printStackTrace();
		}
         Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyname,proxyport));
         URLConnection rulConnection = null;
		try {
			rulConnection = (URLConnection) hurl.openConnection(proxy);
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
         HttpURLConnection httpUrlConnection  =  (HttpURLConnection) rulConnection;	
	try {

	               //httpUrlConnection.setRequestProperty("User-agent","Mozilla/4.0");
	               //httpUrlConnection.setRequestProperty("User-agent","Baiduspider ( http://www.baidu.com/search/spider.htm)");
	               //httpUrlConnection.setRequestProperty("User-agent","Mozilla/4.0"); 
	               //HttpURLConnection.setFollowRedirects(true);	               

	               //String encoded ="Basic "+ new String ( Base64.encodeBase64(new String("lma11:123456a?").getBytes()));               
	               //rulConnection.setRequestProperty( "Authorization", encoded );
	               //String headerkey = "Proxy-Authorization";  
	               //String headerValue = "Basic "+Base64.encodeToString("lma11:123456a?".getBytes(), false);                
	               //rulConnection.setRequestProperty(headerkey, headerValue);  
	               //rulConnection.setRequestProperty( "Proxy-Authorization", "Basic" + Base64.encode("username:password") );               
	               httpUrlConnection.setUseCaches(false);
	               httpUrlConnection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)");
	               httpUrlConnection.setRequestProperty("Accept",
	                       "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-powerpoint, application/vnd.ms-excel, application/msword, */*");
	               httpUrlConnection.setRequestProperty("Accept-Language", "zh-cn");
	               httpUrlConnection.setRequestProperty("UA-CPU", "x86");
	               httpUrlConnection.setRequestProperty("Accept-Encoding", "gzip");//为什么没有deflate呢
	               httpUrlConnection.setRequestProperty("Content-type", "text/html");
	               httpUrlConnection.setRequestProperty("Connection", "close"); //keep-Alive，有什么用呢，你不是在访问网站，你是在采集。嘿嘿。减轻别人的压力，也是减轻自己。

	               httpUrlConnection.setUseCaches(false);//不要用cache，用了也没有什么用，因为我们不会经常对一个链接频繁访问。（针对程序）
	               httpUrlConnection.setDoOutput(true);
	               httpUrlConnection.setDoInput(true);
	               
	               httpUrlConnection.setConnectTimeout(8000);
	               httpUrlConnection.setReadTimeout(8000); 
	               httpUrlConnection.connect();
	  			} catch(Exception e){
	  		        System.out.println("getConnStatus "+proxyname+":" + proxyport + " is not aviable");//异常IP
	  				}
	               code = new Integer(httpUrlConnection.getResponseCode()).toString();
	               String message = httpUrlConnection.getResponseMessage();
	               //if(!code.startsWith("2")){
	                    //throw new Exception("ResponseCode is not begin with 2,code="+code);
	               //}
	               //log.info("连接"+url+"正常");
	           	return code;
	}
	
    public static String webclientproxytest(String url,String proxyname,int proxyport) {  
        URL url1 = null;
        boolean proxyok=false;
        int verifynum=0;
        HtmlPage page = null;
        try {
        url1 = new URL(url);
        } catch (MalformedURLException e) {
        System.out.println("url invalidate");
        }
        WebClient webClient = new WebClient(BrowserVersion.CHROME);        
        ProxyConfig proxyConfig = new ProxyConfig(proxyname,proxyport);  
        try {        	
            webClient.getOptions().setProxyConfig(proxyConfig);        
            final IncorrectnessListener il = new MyIncorrectnessListener();
            webClient.setIncorrectnessListener(il);        
            webClient.getOptions().setTimeout(8000);
        	webClient.getOptions().setJavaScriptEnabled(true);
            webClient.getOptions().setCssEnabled(false);
            webClient.getOptions().setThrowExceptionOnScriptError(false);  
            webClient.getOptions().setThrowExceptionOnFailingStatusCode(false); 
            webClient.setJavaScriptTimeout(8000);
            webClient.setAjaxController(new NicelyResynchronizingAjaxController());
            page = (HtmlPage)webClient.getPage(url);            
        } catch (Exception e) {
        System.out.println("webclientproxytest ip " + proxyname + " is not aviable");//异常IP
        }
        String verifyStr=page.getTitleText();
        if(page!=null){
        System.out.println("数据情况："+verifyStr+" 长度："+verifyStr.indexOf("not"));
        // System.out.println(s);

        if (page.getTitleText().length()> 0&&verifyStr.indexOf("not")<1) {//有效IP
        verifynum=page.getTitleText().length();
        System.out.println("webclientproxytest "+proxyname + ":"+proxyport+"verifynum:"+verifynum+ ",test is ok");
        proxyok=true;
        }	
        }
        return verifyStr;
        }    
	
	
    
}
